package MultiThreaded;

import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;


/**
 * The PageRetriever class turns the source into a string so that it can be
 * later processed by the PageParser.
 * 
 * @author zsyed
 * 
 */
public class PageRetriever implements Runnable {
	/**
	 * Used to check if URL has already been processed.
	 */
    private volatile boolean is_waiting = false;
	private List<String> my_done;
	private  BlockingQueue<String> my_sources;
	private BlockingQueue<String> my_completed_sources;
	private BlockingQueue<URL> my_downloaded_sources;
	public PageRetriever(BlockingQueue<String> sources,
			BlockingQueue<String> completed_sources,
			BlockingQueue<URL> downloaded_sources) {
		my_done = new ArrayList<String>();
		my_sources = sources;
		my_completed_sources = completed_sources;
		my_downloaded_sources = downloaded_sources;
	}

	/**
	 * The thread task is to call retrievePage() until the page limit has been
	 * reached.
	 */
	@Override
	public void run() {
		while (MultiThreadSetup.page_counter < MultiThreadSetup.page_limit) {

	        
			retrievePage();

		}
		my_completed_sources.add("POISON");
		//System.out.println("Finished retrieval entirely");
	}

	/**
	 * Retrieves the source and adds it to a queue so PageParser can use it.
	 */
	private void retrievePage() {
		URL source;
		URL temp_source = null;
		String url;
        is_waiting = true;
		if (!my_sources.isEmpty()) { // can't do this if sources is empty
			try {
			    is_waiting = true;
				url = my_sources.take();
				is_waiting = false;

		        //System.out.println("Starting page retrieval!");
			} catch (InterruptedException e2) {
				return;
			} // get next item from sources<string>
			
			if (!url.startsWith("http://")) {
				url = "http://"+ url;
			}
			if(isURL(url)) {
				try {
					temp_source = new URL(url);
				} catch (Exception e1) {
					System.err.println("Bad url");
				}
				if (temp_source != null && 
						!(my_done.contains(url))) {// also have to check if it hasn't already been processed
					source = temp_source;
					my_done.add(url);
					my_completed_sources.add(url);
					my_downloaded_sources.add(source);
					MultiThreadSetup.page_counter++; 
			        //System.out.println("Finishings page retrieval!");
				}
			}
		}
		else {
			try {
				Thread.sleep(10);
			} catch (InterruptedException e) {
				return;
			}
		}

	}
	/** helper class to check the validity of a url using url patterns matching */
	private boolean isURL(String url) {
		// Assigning the url format regular expression
		String urlPattern = "^http(s{0,1})://[a-zA-Z0-9_/\\-\\.]+\\.([A-Za-z/]{2,5})[a-zA-Z0-9_/\\&\\?\\=\\-\\.\\~\\%]*";
		return url.matches(urlPattern);
	}
	/**
	 * Returns whether the pageretriever is waiting for the pageparser to put something in its 
	 * queue.
	 * @return
	 */
    public boolean getIsWaiting(){
        return is_waiting;
    }
}
